import requests
from fake_useragent import UserAgent
import time

class MySpider(object):
    def __init__(self):
        a=1

    def getHeader(self):
        ua = UserAgent()
        headers = {'User-Agent': ua.random}
        return headers

    def getContent(self,url,retryTime=5,timeOut=10):
        headers = self.getHeader()
        flag = False
        cnt = 0;
        content = ''
        while flag == False and cnt < retryTime:
            status_code = 403
            try:
                response = requests.get(url, headers=headers, timeout=timeOut)
                status_code = response.status_code

                content = response.content

            except requests.exceptions.ConnectTimeout:
                print("请求超时1")
                flag = False
            except requests.exceptions.Timeout:
                print("请求超时2")
                flag = False
            except requests.exceptions.ConnectionError:
                print("请求超时3")
                flag = False
            except requests.exceptions.ChunkedEncodingError:
                print("ChunkedEncodingError")
                flag = False

            if status_code != 200:
                flag = False
            else:
                flag = True
            cnt += 1
            if flag==False:
                print("%s %s" % (status_code, url))
                time.sleep(1)


        if flag:
            return (flag, content)
        else:
            return (flag, "")
